import re
import pymysql
import requests
from bs4 import element
from bs4 import BeautifulSoup

pcontent = '''   <div style="position:relative;height:90px;">
  <a href="http://gb.corp.163.com/gb/legal.html" class="ad_hover_href"></a>
<a class="right" href="http://g.163.com/a?CID=47495&Values=2563197190&Redirect=http://www.qdmzgsm.com/1703/YM1/"><img src="http://img1.126.net/channel3/026190_40090_161222.jpg" width="400" height="90" border="0" alt="牛股王" /></a>
</div>'''

def getNetEaseNewsDetail(url):
    if not ('.html' in url):
        return None
    newsDetail = {}
    try:
        res = requests.get(url)
        soul = BeautifulSoup(res.text, 'html5lib')
        newsDetail['docurl'] = url
        newsDetail['title'] = soul.select('.post_content_main h1')[0].text
        newsDetail['sourcefrom'] = soul.select('#ne_article_source')[0].text
        divArray = soul.select("#endText")
        pcontent = '';
        for div in divArray:
            # [s.extract() for s in div('div')]
            for item in div.contents:
                if (not isinstance(item, element.NavigableString)) and len(item.select('.ad_hover_href')) > 0:
                    pcontent += ""
                elif ('<pre' in str(item) and '</pre>' in str(item)) or ('special_tag_wrap' in str(item)):
                    pcontent += ""
                elif 'AD200x300' in str(item) or 'ep-source' in str(item):
                    pcontent += ""
                elif 'alt="牛股王"' in str(item):
                    pcontent += ""
                elif not isinstance(item, element.NavigableString) and len(item.select('img')) > 0:
                    pcontent += getMatchString(str(item))
                else:
                    pcontent += str(item)
        newsDetail['content'] = pcontent
        print('正常: ', newsDetail['title'])
    except Exception as e:
        print('!有异常', e)
        newsDetail = None
    return newsDetail


def getMatchString(pcontent):
    #过滤广告标签
    pattern = re.compile("alt=\"牛股王\"")
    match = pattern.search(pcontent)
    if match:
        pcontent = ''
        return pcontent
    # 过滤height
    pattern = re.compile("height=\"(.+?)\"")
    match = pattern.search(pcontent)
    if match:
        pcontent = pcontent.replace(match.group(), "")
    # 过滤widht
    pattern = re.compile("width=\"(.+?)\"")
    match = pattern.search(pcontent)
    if match:
        pcontent = pcontent.replace(match.group(), "")
    # 过滤style标签
    pattern = re.compile("style=\"(.+?)\"")
    match = pattern.search(pcontent)
    if match:
        pcontent = pcontent.replace(match.group(), "")
    # 过滤style标签
    pattern = re.compile("style=\'(.+?)\'")
    match = pattern.search(pcontent)
    if match:
        pcontent = pcontent.replace(match.group(), "")
    return pcontent

data = getNetEaseNewsDetail("http://money.163.com/17/0823/12/CSHANI4J00258152.html")
print(data['content'])
